In [1]:
# import packages
%matplotlib inline
%pylab inline
pylab.rcParams['figure.figsize'] = (10, 6)
import numpy as np
import matplotlib.pyplot as plt
import bokeh.plotting as bp
import matplotlib.animation as animation
from bokeh.models import WheelZoomTool,ResetTool,PanTool
from JSAnimation import IPython_display
W = 590
H = 350
bp.output_notebook()
In [2]:
#plotting the loss function
x = np.linspace(-2.,2.,500)
y = x**4 - 2*x**2 + 2*x
i = np.argmin(y)
print("Global minimun of loss function is at point (%.2f,%.2f)"%(x[i],y[i]))
fig = plt.figure()
ax = plt.axes()
plt.plot(x,y)
plt.title("Plot of the loss function $x^{4}-2x^{2}+2x$")
plt.grid(True)
ax.annotate('Min @ '+str(round(x[i],2)),xy=(x[i],y[i]),\
xytext=(x[i],y[i]+2.5),\
arrowprops=dict(facecolor='red',shrink=0.02))
Out[2]:
Testing interactive plots
In [3]:
x = np.linspace(-15,21,100)
y = x**2-6*x+5
TOOLS =[WheelZoomTool(),ResetTool(),PanTool()]
s1 = bp.figure(width=W,plot_height=H,
title='Local minimum of function',
tools=TOOLS)
s1.line(x,y,color='navy',alpha=0.5,line_width=2)
s1.circle(3,-4,size=10,color='orange')
s1.title_text_font_size='16pt'
s1.yaxis.axis_label_text_font_size='14pt'
s1.xaxis.axis_label_text_font_size='14pt'
bp.show(s1)
Out[3]:
In [7]:
old_min = 0
temp_min = 15
step_size = 0.01
precision = 0.001
def f_derivative(x):
return 2*x -6
mins = []
cost = []
while abs(temp_min - old_min) > precision:
old_min = temp_min
gradient = f_derivative(old_min)
move = gradient * step_size
temp_min = old_min - move
cost.append((3-temp_min)**2)
mins.append(temp_min)
def init():
line.set_data([],[])
return line,
def animate(i):
x_n = mins[0::10][i]
y_n = x_n**2-6*x_n+5
line.set_data(x_n,y_n)
return line,
fig = plt.figure()
ax = plt.axes(xlim=(-15,21),ylim=(-50,350))
ax.plot(x,y,linewidth=3)
line, = ax.plot([],[],"D",markersize=12)
animation.FuncAnimation(fig,animate,init_func=init,
frames=len(mins[0::10]),interval=200)
Out[7]:
getGrad function get a point,number of samples and outputs an array of values with Gaussian noise drawn from a standard normal distribution based on the formula:
In [32]:
def getGrad(p,n):
v = 4*p**3-4*p+2
arr = np.ones(n)*v
noise = np.random.normal(loc=0,scale=1,size=n)
return arr+noise
def batchGradientDescent(init,sampleSize,iterations,learningRate=0.1):
ws = np.zeros(iterations+1)
ws[0] = init
for i in range(iterations):
grads = getGrad(ws[i],sampleSize)
ws[i+1] = ws[i] - learningRate*grads.mean()
return ws
def miniBatchGradientDescent(init,sampleSize,iterations,
learningRate=0.1,batch=1):
ws = np.zeros(iterations+1)
ws[0] = init
for i in range(iterations):
grads = getGrad(ws[i],sampleSize)
ch = np.random.choice(grads,size=batch)
ws[i+1] = ws[i] - learningRate*ch.mean()
return ws
In [81]:
resBatch = batchGradientDescent(1.5,100,100,learningRate=0.1)
resMini = miniBatchGradientDescent(1.5,100,100,batch=10,learningRate=0.1)
resOnline = miniBatchGradientDescent(1.5,100,100,batch=1,learningRate=0.1)
lossBatch = np.abs(resBatch-(-1.19))
lossMini = np.abs(resMini-(-1.19))
lossOnline = np.abs(resOnline-(-1.19))
In [115]:
it = np.arange(1,102)
plt.plot(it,lossBatch,linewidth=2.)
plt.plot(it,lossMini,linewidth=2.)
plt.plot(it,lossOnline,linewidth=2.)
lines = plt.gca().get_lines()
plt.legend(lines,['Batch_Gradient_Loss',\
'MiniBatch_Gradient_Loss',\
'Online_Gradient_Loss'])
plt.title('Loss of gradient versions in one epoch',color='navy')
plt.xlabel('Loss',color='navy')
plt.ylabel('Iterations',color='navy')
grid(True)
Graph of the three gradient descent versions with interactive tools
In [121]:
TOOLS =[WheelZoomTool(),ResetTool(),PanTool()]
s1 = bp.figure(width=W,plot_height=H,
title='Loss of gradient versions in one epoch',
tools=TOOLS)
xs=[it,it,it]
ys=[lossBatch,lossMini,lossOnline]
#s1.multi_line(xs,ys,line_color=['navy','green','red'],line_width=2)
s1.line(it,lossBatch,line_color='navy',legend='Batch_Gradient_Loss',
line_width=2.)
s1.line(it,lossMini,line_color='green',legend='Mini_Batch_Gradient_Loss',
line_width=2.)
s1.line(it,lossOnline,line_color='red',legend='Online_Gradient_Loss',
line_width=2.)
s1.xaxis.axis_label="Iterations/One Epoch"
s1.yaxis.axis_label="Loss"
s1.title_text_font_size='16pt'
s1.yaxis.axis_label_text_font_size='14pt'
s1.xaxis.axis_label_text_font_size='14pt'
bp.show(s1)
Out[121]:
Animated plot of Batch Gradient Descent Loss for 1 epoch
In [110]:
fig = plt.figure()
ax = plt.axes(xlim=(0,102),ylim=(floor(lossBatch.min()),ceil(lossBatch.max())))
line, = ax.plot([],[],lw=2.,color='navy')
plt.grid(True)
x=[]
y=[]
n=10
def init():
line.set_data([],[])
return line,
#lossBatch (y) and iterations(x)
def animate(i):
x.append(np.linspace(i,i+1,n))
y.append(np.linspace(lossBatch[i],lossBatch[i+1],n))
line.set_data(x,y)
return line,
animation.FuncAnimation(fig,animate,np.arange(0,len(it)-1),
init_func=init,
interval=50,blit=True,repeat=False)
Out[110]:
Animated plot of Mini-Batch Gradient Descent Loss for 1 epoch
In [123]:
fig = plt.figure()
ax = plt.axes(xlim=(0,102),ylim=(floor(lossMini.min()),ceil(lossMini.max())))
line, = ax.plot([],[],lw=2.,color='green')
plt.grid(True)
x=[]
y=[]
n=10
def init():
line.set_data([],[])
return line,
#lossBatch (y) and iterations(x)
def animate(i):
x.append(np.linspace(i,i+1,n))
y.append(np.linspace(lossMini[i],lossMini[i+1],n))
line.set_data(x,y)
return line,
animation.FuncAnimation(fig,animate,np.arange(0,len(it)-1),
init_func=init,
interval=50,blit=True,repeat=False)
Out[123]:
Animated plot of Online Gradient Descent Loss for 1 epoch
In [124]:
fig = plt.figure()
ax = plt.axes(xlim=(0,102),ylim=(floor(lossOnline.min()),ceil(lossOnline.max())))
line, = ax.plot([],[],lw=2.,color='red')
plt.grid(True)
x=[]
y=[]
n=10
def init():
line.set_data([],[])
return line,
#lossBatch (y) and iterations(x)
def animate(i):
x.append(np.linspace(i,i+1,n))
y.append(np.linspace(lossOnline[i],lossOnline[i+1],n))
line.set_data(x,y)
return line,
animation.FuncAnimation(fig,animate,np.arange(0,len(it)-1),
init_func=init,
interval=50,blit=True,repeat=False)
Out[124]:
In [122]:
lossBatch
Out[122]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: